package com.spider.silence.downloader;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.spider.silence.abstarct.downloader.AbstractLoginDownLoader;
import com.spider.silence.proxy.ProxyPool;
import com.spider.silence.task.TencentWeiBoModuleTask;
import com.spider.silence.task.TencentWeiBoTask;
import com.spider.silence.task.manager.CommonTaskManager;
import com.spider.silence.utils.*;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import org.dom4j.Element;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import org.openqa.selenium.By;
import org.openqa.selenium.Cookie;
import org.openqa.selenium.chrome.ChromeDriver;
import org.openqa.selenium.support.ui.ExpectedConditions;
import org.openqa.selenium.support.ui.WebDriverWait;
import us.codecraft.webmagic.Page;

import java.io.File;
import java.lang.reflect.InvocationTargetException;
import java.net.URLDecoder;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.concurrent.*;

/**
 * @author liyuanguo
 * @version V1.0
 * @time 2017/2/7 11:20
 * @description
 */
public class TencentWeiBoDownloader extends AbstractLoginDownLoader implements Cloneable{

    private Logger logger = Logger.getLogger(TencentWeiBoDownloader.class);
    private String qi;
    private String pwd;
    private Map<String,File> files=null;

    public Map<String,File> getFiles(){

        return files;
    }
    public TencentWeiBoDownloader(String qi, String pwd){
        this.qi = qi;
        this.pwd = pwd;
    }

    public TencentWeiBoDownloader(){

    }

    @Override
    protected HashSet<Cookie> login() {
        ChromeDriver driver = ChromeDriverUtils.getDriver();
        try {
            driver.get("http://t.qq.com/");
//            driver.manage().window().maximize();
            driver.switchTo().frame("login_div").findElement(By.cssSelector("#switcher_plogin")).click();
            WebDriverWait loginFormWait = new WebDriverWait(driver, 20000);
            driver.manage().timeouts().implicitlyWait(10, TimeUnit.SECONDS);
            while (true) {
                try {
                    Thread.sleep(500);
                } catch (InterruptedException e) {
                    logger.error(e.getMessage(),e);
                    e.printStackTrace();
                    throw e;
                }
                if (driver.findElement(By.cssSelector("#u")).isEnabled()) {
                    loginFormWait.until(ExpectedConditions.visibilityOfElementLocated(By.cssSelector("#u")))
                            .sendKeys(qi);
                    break;
                }
            }
            while (true) {
                try {
                    Thread.sleep(500);
                } catch (InterruptedException e) {
                    logger.error(e.getMessage(),e);
                    e.printStackTrace();
                    throw e;
                }
                if (driver.findElement(By.cssSelector("#p")).isEnabled()) {
                    loginFormWait.until(ExpectedConditions.visibilityOfElementLocated(By.cssSelector("#p")))
                            .sendKeys(pwd);
                    break;
                }
            }
            loginFormWait.until(ExpectedConditions.visibilityOfElementLocated(By.cssSelector("#login_button"))).click();
            long start=System.currentTimeMillis();
            long end=System.currentTimeMillis();

            while (true&&(end-start)<20000) {
                end=System.currentTimeMillis();
                try {
                    Thread.sleep(500);
                } catch (InterruptedException e) {
                    logger.error(e.getMessage(),e);
                    e.printStackTrace();
                    throw e;
                }
                //System.out.println("当前url长度:"+driver.getCurrentUrl()+":长度:"+driver.getCurrentUrl().length());
                if (driver.getCurrentUrl().length()>16) {
                    int flag=(int)Math.floor(Math.random()*1000);
                    if(flag<300){
                        try {
                            try {
                                Thread.sleep((int)Math.floor(8000));
                            } catch (InterruptedException e) {
                                logger.error(e.getMessage(),e);
                                e.printStackTrace();
                                throw e;
                            }

                        } catch (Exception e) {
                            logger.error(e.getMessage(),e);
                            e.printStackTrace();
                            throw e;
                        }
                    }
                    try {
                        Thread.sleep(1000);
                    } catch (InterruptedException e) {
                        logger.error(e.getMessage(),e);
                        e.printStackTrace();
                        throw e;
                    }
                    break;
                }
            }
            try {
                Thread.sleep(1000);
            } catch (InterruptedException e) {
                logger.error(e.getMessage(),e);
                e.printStackTrace();
            }
            if(end-start<20000){
                try {
                    cookies = (HashSet<Cookie>) driver.manage().getCookies();
                    //System.out.println("coookie内容："+cookies);
                } catch (NullPointerException e) {
                    logger.error("登陆错误",e);
                    throw new Exception("登陆错误");
                }
            }else{
                logger.error(qi+":不能登录");
                throw new Exception(qi+":不能登录");
            }
            return  cookies;
        }catch(Exception e){
            logger.error(e.getMessage(),e);
            e.printStackTrace();
        } finally {
            driver.close();
            driver.quit();
        }
        return null;
    }

    public void generateTask() {
        HashSet<String> searchUrlSet=new HashSet<String>();
        searchUrlSet=getSearchUrlSet();
        if(searchUrlSet.size()>0){
            for(String moduleUrl:searchUrlSet){
                //System.out.println("moduleurl:"+moduleUrl);
                    if(null!=moduleUrl&&!"".equals(moduleUrl)){
                        boolean flag = true;
                        long startTime = System.currentTimeMillis();
                        long nowTime = startTime;
                        while (flag && nowTime - startTime < Long.valueOf(PropertiesUtil.getVal("TencentWeiBo.TimeOut"))) {
                            try {
                                Document document = null;
                                Page pageContent = null;
                                String getDataUrl="";
                                getDataUrl=moduleUrl;
                                pageContent = this.getHtml(getDataUrl, null);
                                if (null == pageContent) {

                                } else if (pageContent.getStatusCode() == 0) {

                                } else if (String.valueOf(pageContent.getStatusCode()).startsWith("1")) {

                                } else if (pageContent.getStatusCode() == 200) {
                                    flag = false;
                                    document = pageContent.getHtml().getDocument();
                                    Elements eles=document.select("#talkList");
                                    //System.out.println("内容："+eles);
                                    Elements elements=document.select("#talkList").select("li");

                                    //获取每个模块下的第一页所有帖子的url、组装任务
                                    for(int i=0;i<elements.size();i++){
                                        String tid=elements.get(i).attr("id");
                                        String forumUrl="http://t.qq.com/p/t/"+tid;
                                        //System.out.println("帖子url："+forumUrl);
                                        TencentWeiBoTask task=new TencentWeiBoTask(forumUrl);
                                        CommonTaskManager.add("tencentWeiBo", task);
                                    }

                                } else if (String.valueOf(pageContent.getStatusCode()).startsWith("3")) {

                                } else if (String.valueOf(pageContent.getStatusCode()).startsWith("4")) {

                                } else if (String.valueOf(pageContent.getStatusCode()).startsWith("5")) {

                                } else {

                                }

                            } catch (Exception e1) {
                                logger.error(e1.getMessage(), e1);
                            }finally{
                                nowTime = System.currentTimeMillis();
                            }
                        }
                    }
            }
        }

    }

    public HashSet<String> getSearchUrlSet(){
        org.dom4j.Document document = XmlUtil.getConf("META-INF/TencentWeibo.xml");
        List<Element> elements=document.getRootElement().element("condition-list").elements();
        HashSet<String> searchUrlSet=new HashSet<String>();
        for(int i=0;i<elements.size();i++){
            try {
                Element element=elements.get(i);
                String condition=element.element("value").getText();
                String type=element.element("type").getText();
                if("1".equals(type)){
                    String moduleUrl="http://search.t.qq.com/index.php?k="+condition;
                    TencentWeiBoModuleTask task=new TencentWeiBoModuleTask(moduleUrl);
                    searchUrlSet.add(moduleUrl);
                }else if("2".equals(type)){
                    String moduleUrl="http://t.qq.com/"+condition;
                    TencentWeiBoModuleTask task=new TencentWeiBoModuleTask(moduleUrl);
                    searchUrlSet.add(moduleUrl);
                }
            } catch (Exception e) {
                logger.error(e.getMessage(),e);
                continue;
            }
        }
        return searchUrlSet;
    }

    public static void main(String[] args) {
        TencentWeiBoDownloader dow=new TencentWeiBoDownloader();
        dow.crawlItem("http://t.qq.com/p/t/515315073334689");
    }

    public JSONObject crawlItem(String url) {
        files=new ConcurrentHashMap<>();
        JSONObject item = new JSONObject();
        JSONObject data = new JSONObject();
        int page = 1;
        //System.out.println("爬取的：" + url);
        try {
            // Document document=get(url,true);
            Document document = null;
            boolean flag = true;
            long startTime = System.currentTimeMillis();
            long nowTime = startTime;
            while (flag && nowTime - startTime < Long.valueOf(PropertiesUtil.getVal("TencentWeiBo.TimeOut"))) {
                try {
                    Page pageContent = null;
                    //正确方式pageContent = this.getHtml(url, ProxyPool.proxy());
                    //以下测试用的
                    pageContent = this.getHtml(url, null);
                    if (null == pageContent) {

                    } else if (pageContent.getStatusCode() == 0) {

                    } else if (String.valueOf(pageContent.getStatusCode()).startsWith("1")) {

                    } else if (pageContent.getStatusCode() == 200) {
                        try {
                            //System.out.println("爬取言论第一页:" + url + "状态码：" + pageContent.getStatusCode());
                            String id=url.substring(20,url.length());
                            document = pageContent.getHtml().getDocument();
                            String title = document.title();
                            org.jsoup.nodes.Element pageInfo = document.select("#main").get(0);
                            String content = "";

                            //发请求获取回复板块数据json
                            List<String> pageUrlList=new ArrayList<String>();
                            Document replyDoc=null;
                            boolean flag1=true;
                            long startTime1=System.currentTimeMillis();
                            long nowTime1=startTime1;
                            while(flag1&&nowTime1-startTime1<Long.valueOf(PropertiesUtil.getVal("TencentWeiBo.TimeOut"))){
                                try {
                                    //String reply_url="http://api.t.qq.com/old/message.php?id=%s&format=1&p=1&select=1&apiType=14";
                                    String reply_url="http://api.t.qq.com/old/message.php?id=%s&format=1&filter=6&select=3&p=1&apiType=14";
                                    reply_url=String.format(reply_url,new Object[]{id});
                                    //此处请求需要带cookie
                                    String replyContent= this.getString(reply_url,null);
                                    if(null!=replyContent&&!"".equals(replyContent)) {
                                        JSONObject replyJson= JSON.parseObject(replyContent);
                                        String  replyContentFist=replyJson.getString("info");
                                        if(!"".equals(replyContentFist)){
                                            flag1=false;
                                            String replyHtml="<html>  <body>"+replyContentFist+" </body> </html>";
                                            replyDoc= Jsoup.parse(replyHtml);

                                        }
                                    }

                                    nowTime1=System.currentTimeMillis();
                                } catch (Exception e1) {
                                    //System.out.println("异常信息："+e1.getMessage());
                                    logger.error(e1.getMessage(),e1);
                                }
                            }


                            if (null != pageInfo.select(".msgBox") && pageInfo.select(".msgBox").size() > 0&&!"".equals(pageInfo.select(".msgBox").get(0).html())&&null!=replyDoc) {

                                // 爬取第一页上帖子的内容
                                String reply_count_string="0";
                                if(null!=replyDoc.select(".tabStyle1").select("li")&&replyDoc.select(".tabStyle1").select("li").size()>0){
                                    reply_count_string=replyDoc.select(".tabStyle1").select("li").get(2).text();
                                    if("评论".equals(reply_count_string)){
                                        reply_count_string="0";
                                    }else{
                                        reply_count_string=reply_count_string.substring(3,reply_count_string.length()-1);
                                    }
                                }
                                int reply_count = Integer.parseInt(reply_count_string);// 回复量
                                //System.out.println("回复数目："+reply_count);
                                String reproduce_count_string="0";
                                if(null!=replyDoc.select(".tabStyle1").select("li")&&replyDoc.select(".tabStyle1").select("li").size()>0){
                                    reproduce_count_string=replyDoc.select(".tabStyle1").select("li").get(1).select("a").text();
                                    if("转播".equals(reproduce_count_string)){
                                        reproduce_count_string="0";
                                    }else{
                                        reproduce_count_string=reproduce_count_string.substring(3,reproduce_count_string.length()-1);
                                    }
                                }
                                //System.out.println("转发数目:"+reproduce_count_string);

                                Elements pages=replyDoc.select(".pages").select("a");

                                if(null!=pages&&pages.size()>2){
                                    for(int i=2;i<pages.size()-1;i++){
                                        pageUrlList.add(pages.get(i).attr("href"));
                                    }
                                    page=page+pageUrlList.size();
                                }

                                String publish_time = "";// 发布时间
                                String pubTimeString=pageInfo.select("#orginCnt").select(".orginCnt").select(".msgBox").select(".pubInfo").select(".left").select(".time").get(0).attr("rel");
                                SimpleDateFormat localSimpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
                                Date date = new Date(Long.parseLong(pubTimeString+"000"));
                                publish_time =localSimpleDateFormat.format(date);
                                //System.out.println("发布时间："+publish_time);
                                content = pageInfo.select(".msgBox").get(0).html();// 发布内容
                                Elements imgElements=pageInfo.select(".msgBox").get(0).select(".tl_imgGroup_item");
                               Elements imgElements2=pageInfo.select(".msgBox").get(0).select(".mediaWrap");
                                JSONArray img_urls=new JSONArray();
                                if(imgElements.size()>0){
                                    for(int imgIndex=0;imgIndex<imgElements.size();imgIndex++){
                                        org.jsoup.nodes.Element imgElement=imgElements.get(imgIndex).select("img").get(0);
                                        String img_url=imgElement.absUrl("crs");
                                        img_urls.add(img_url);
                                    }
                                }
                                if(imgElements2.size()>0){
                                    Elements imgElement=imgElements2.get(0).select(".pic").get(0).select("img");
                                    System.out.println(imgElements2.get(0).select(".pic"));
                                    for(int imgIndex=0;imgIndex<imgElement.size();imgIndex++){
                                        String img_url=imgElement.get(imgIndex).absUrl("crs");
                                        img_urls.add(img_url);
                                    }
                                }
                                //System.out.println("发布内容：" + content);
                                String publish_user =pageInfo.select(".m_profile").select(".tit").select("a").get(0).text();
                                //System.out.println("用户:"+publish_user);
                                String publish_user_id_url = pageInfo.select(".m_profile").select(".tit").select("a").get(0).attr("href");// 发布者的ID
                                String publish_user_id=publish_user_id_url.substring(16,publish_user_id_url.length());
                                //System.out.println("用户id："+publish_user_id);
                                String publish_user_photo = pageInfo.select(".m_profile").select(".avatar").select("img").get(0).attr("src");
                                //System.out.println("用户头像："+publish_user_photo);
                                //System.out.println("帖子id："+id);

                                 try {
                                     String like ="0";
                                     if(null!=replyDoc.select(".tabStyle1").select("li")&&replyDoc.select(".tabStyle1").select("li").size()>0){
                                         like=replyDoc.select(".tabStyle1").select("li").get(3).select("a").text();
                                     }
                                     if(like.contains("认证用户")){
                                         like="0";
                                     }else if("赞".equals(like)){
                                         like="0";
                                     }else if(null!=like&&!"".equals(like)){
                                         like=like.substring(2,like.length()-1);
                                     }else{
                                         like="0";
                                     }
                                     //System.out.println("点赞数："+like);

                                    if (!"".equals(like) && like != null) {
                                        int like_count = Integer.parseInt(like);
                                        data.put("like_count", like_count);
                                    } else {
                                        data.put("like_count", 0);
                                    }
                                } catch (Exception e) {
                                     logger.error(e.getMessage(),e);
                                     data.put("like_count", 0);
                                }

                                data.put("reply_count", reply_count);
                                //data.put("read_count", read_count);
                                data.put("title", (title));
                                data.put("url", url);
                                data.put("publish_time", publish_time);
                                Date now = new Date();
                                SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
                                data.put("spider_time", sdf.format(now));
                                data.put("reproduce_count", reproduce_count_string);
                                data.put("content", FileUtils.filterImg(content));
                                data.put("img_urls",img_urls);
                                data.put("publish_user_id", publish_user_id);
                                data.put("publish_user", (publish_user));
                                data.put("publish_user_photo", publish_user_photo);
                                data.put("id", id);

                                long tsp= DateUtils.dateToTimestamp( DateUtils.parseDate(publish_time)).getTime();
                                String filePrefix="TencentWeiBo";//文件前缀
                                String fileName=filePrefix+"_"+tsp+"_"+publish_user_id+"_"+id;
                                String publishDate=DateUtils.formatDate(DateUtils.parseDate(publish_time)).replace("-","-");
                                String speechesId="TencentWeiBo_speeches_"+publishDate+"_"+fileName;
                                File file= FileUtils.createTempFile(speechesId+"_pageNo_1",document.html().toString());
                                files.put(speechesId+"_pageNo_1",file);
                                //帖子首页的文件

                               Elements replyEntities = replyDoc.select(".clear").select("li[id$='_relay']");
                                JSONArray reply_nodes = new JSONArray();
                                // System.out.println("回复："+replyEntities);

                                // 爬取第一页的回复
                                for (int i = 0; i < replyEntities.size(); i++) {
                                    try {
                                        org.jsoup.nodes.Element replyInfo = replyEntities.get(i).select(".msgBox").get(0);
                                        //System.out.println("回复用户信息："+replyUserInfo);
                                        JSONObject reply = new JSONObject();
                                        String replyUser = "";
                                        String replyUserId = "";
                                        replyUser=replyInfo.select(".msgCnt").select("a").get(0).text();
                                        String replyUserIdString=replyInfo.select(".msgCnt").select("a").get(0).attr("href");
                                        if(null!=replyUserIdString&&!"".equals(replyUserIdString)){
                                            replyUserId=replyUserIdString.substring(16,replyUserIdString.length());
                                        }
                                        if("".equals(replyUser)&&"".equals(replyUserId)){
                                            continue;
                                        }
                                        //System.out.println("用户id："+replyUserId);
                                        //replyUser=replyInfo.select(".msgCnt").select("a").get(0).text();
                                        //System.out.println("回复用户名："+replyUser);

                                        String replyUserPhoto = replyEntities.get(i).select(".userPic").select("img").get(0).attr("src");
                                        //System.out.println("回复者头像："+replyUserPhoto);
                                        String replyContent = replyInfo.select(".msgCnt").select(".content").get(0).html();
                                        //System.out.println("回复内容："+replyContent);
                                        String replyTimeString=replyInfo.select(".pubInfo").select(".cNote").select("a").attr("rel");
                                        Date replyDate = new Date(Long.parseLong(replyTimeString+"000"));
                                        String replyTime =localSimpleDateFormat.format(replyDate);
                                        //System.out.println("回复时间："+replyTime);
                                        //String replyUrl = url + "#" + replyEntities.get(i).id();
                                        // System.out.println("回复url："+replyUrl);
                                        JSONArray reply_img_urls=new JSONArray();
                                        Elements replyImgElements=replyInfo.select(".msgCnt").select(".content").get(0).select("img");
                                        for(int imgIndex=0;imgIndex<replyImgElements.size();imgIndex++){
                                            org.jsoup.nodes.Element imgElement=replyImgElements.get(imgIndex);
                                            String img_url=imgElement.absUrl("crs");
                                            reply_img_urls.add(img_url);
                                        }
                                        reply.put("publish_user", replyUser);
                                        reply.put("publish_user_id", replyUserId);
                                        reply.put("publish_user_photo", replyUserPhoto);
                                        reply.put("content", FileUtils.filterImg(replyContent));
                                        reply.put("img_urls",reply_img_urls);
                                        reply.put("publish_time", replyTime);
                                        //reply.put("url", replyUrl);
                                        reply.put("parent_id", id);
                                        reply.put("ancestor_id", id);
                                        reply_nodes.add(reply);
                                    } catch (Exception e) {
                                        logger.error(e.getMessage(),e);
                                        continue;// 因为禁言无法获取的情况
                                    }

                                }

                                JSONArray replys = new JSONArray();
                                List<Map<String, Object>> list = new ArrayList<Map<String, Object>>();

                                // 爬取2页之后每页的回复
                                if (page >= 2) {
                                    int threadNum = Integer.valueOf(PropertiesUtil.getVal("TencentWeiBo.ReplyThreadCount"));
                                    List<FutureTask<JSONArray>> futureTasks = new ArrayList<FutureTask<JSONArray>>();
                                    // 线程池 初始化十个线程 和JDBC连接池是一个意思 实现重用
                                    ExecutorService executorService = Executors.newFixedThreadPool(threadNum);
                                    try {
                                        ArrayList<ArrayList<Integer>> splitPages = CommonUtils.splitPages(2, page, threadNum);
                                        for (int i = 0; i < splitPages.size(); i++) {
                                            TencentWeiBoDownloader downLoader = this.cloneDownLoader();

                                            ReplyCrawler callable = new ReplyCrawler(downLoader, splitPages.get(i), url, id,pageUrlList,speechesId);
                                            FutureTask<JSONArray> futureTask = new FutureTask<JSONArray>(callable);
                                            futureTasks.add(futureTask);
                                            executorService.submit(futureTask);
                                        }
                                        int count = 0;

                                        for (FutureTask<JSONArray> futureTask : futureTasks) {
                                            JSONArray replyArrays = futureTask.get();
                                            for (int i = 0; i < replyArrays.size(); i++) {
                                                JSONObject object = replyArrays.getJSONObject(i);
                                                replys.add(object);
                                            }
                                        }
                                        if (replys.size() > 0 && replys != null) {
                                            for (int i = 0; i < replys.size(); i++) {
                                                JSONObject reply_object = replys.getJSONObject(i);
                                                reply_nodes.add(reply_object);
                                            }
                                        }
                                        if (reply_nodes.size() > 0 && reply_nodes != null) {
                                            for (int i = 0; i < reply_nodes.size(); i++) {
                                                JSONObject reply_object = reply_nodes.getJSONObject(i);
                                                Map<String, Object> map = reply_object;
                                                list.add(map);
                                            }
                                        }
                                        Collections.sort(list, new Comparator<Map<String, Object>>() {

                                            public int compare(Map<String, Object> o1, Map<String, Object> o2) {
                                                String s1 = o1.get("publish_time").toString();
                                                String s2 = o2.get("publish_time").toString();
                                                int flag = s1.compareTo(s2);
                                                return flag;
                                            }
                                        });
                                    }catch (ExecutionException e) {
                                        logger.error(e.getMessage(),e);
                                        e.printStackTrace();
                                    } finally {
                                        executorService.shutdown();
                                    }
                                } else {
                                    if (reply_nodes.size() > 0 && reply_nodes != null) {
                                        for (int i = 0; i < reply_nodes.size(); i++) {
                                            JSONObject reply_object = reply_nodes.getJSONObject(i);
                                            Map<String, Object> map = reply_object;
                                            list.add(map);
                                        }
                                    }
                                }
                                data.put("reply_nodes", list);
                            } else {

                            }
                        } catch (Exception e) {

                            logger.error(e.getMessage(),e);
                        }finally{
                            flag = false;
                        }

                    } else if (String.valueOf(pageContent.getStatusCode()).startsWith("3")) {

                    } else if (String.valueOf(pageContent.getStatusCode()).startsWith("4")) {

                    } else if (String.valueOf(pageContent.getStatusCode()).startsWith("5")) {

                    } else {

                    }

                } catch (Exception e1) {
                    //System.out.println("异常信息：" + e1.getMessage());
                    logger.error(e1.getMessage(),e1);
                }finally{
                    if(flag==true){
                        item = new JSONObject();
                        data = new JSONObject();
                    }
                    nowTime = System.currentTimeMillis();
                }
            }

        } catch (Exception e) {
            item = new JSONObject();
            data = new JSONObject();
            logger.error(e.getMessage(),e);
        }
        item.put("data", data);
        return item;
    }

    public TencentWeiBoDownloader cloneDownLoader()
            throws InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException {
        TencentWeiBoDownloader loader = new TencentWeiBoDownloader();
        loader.cookies = (HashSet<Cookie>) this.cookies.clone();
        loader.headers = (HashMap<String, String>) this.headers.clone();
        return loader;
    }

    public class ReplyCrawler implements Callable<JSONArray> {
        private TencentWeiBoDownloader downLoader;
        private ArrayList<Integer> pages;
        private String url;
        private String parent_id;
        private List<String> pageUrlList;
        private String speeches_id;

        @Override
        public JSONArray call() throws Exception {
            JSONArray array = new JSONArray();
            for (int k = 0; k < pages.size(); k++) {
                JSONArray arrayPage = new JSONArray();
                try {

                    boolean flag = true;
                    long startTime = System.currentTimeMillis();
                    long nowTime = startTime;
                    while (flag && nowTime - startTime < Long.valueOf(PropertiesUtil.getVal("TencentWeiBo.TimeOut"))) {
                        try {
                            String getUrl="";
                            for(String pageUrl:this.pageUrlList){
                                if(pages.get(k)==Integer.valueOf(getUrlParam(pageUrl).get("p").toString())){
                                    getUrl=pageUrl;
                                    break;
                                }
                            }
                            Document document = null;
                            Page pageContent = null;
                            pageContent = downLoader.getHtml(getUrl, null);
                            if (null == pageContent) {

                            } else if (pageContent.getStatusCode() == 0) {

                            } else if (String.valueOf(pageContent.getStatusCode()).startsWith("1")) {

                            } else if (pageContent.getStatusCode() == 200) {
                                String replyContent=pageContent.getRawText();
                                Document replyDoc=null;
                                if(null!=replyContent&&!"".equals(replyContent)) {
                                    JSONObject replyJson= JSON.parseObject(replyContent);
                                    String  replyContentFist=replyJson.getString("info");
                                    if(!"".equals(replyContentFist)){
                                        String replyHtml="<html>  <body>"+replyContentFist+" </body> </html>";
                                        replyDoc= Jsoup.parse(replyHtml);
                                    }
                                }
                                File file=FileUtils.createTempFile(speeches_id+"_pageNo_"+pages.get(k),replyDoc.html().toString());
                                files.put(speeches_id+"_pageNo_"+pages.get(k),file);

                                Elements replyEntities = replyDoc.select(".clear").select("li[id$='_relay']");
                                JSONArray reply_nodes = new JSONArray();
                                for (int i = 0; i < replyEntities.size(); i++) {
                                    try {
                                        org.jsoup.nodes.Element replyInfo = replyEntities.get(i).select(".msgBox").get(0);
                                        JSONObject reply = new JSONObject();
                                        String replyUser = "";
                                        String replyUserId = "";
                                        replyUser=replyInfo.select(".msgCnt").select("a").get(0).text();
                                        String replyUserIdString=replyInfo.select(".msgCnt").select("a").get(0).attr("href");
                                        if(null!=replyUserIdString&&!"".equals(replyUserIdString)){
                                            replyUserId=replyUserIdString.substring(16,replyUserIdString.length());
                                        }

                                        //System.out.println("用户id："+replyUserId);
                                        //replyUser=replyInfo.select(".msgCnt").select("a").get(0).text();
                                        //System.out.println("回复用户名："+replyUser);
                                        if("".equals(replyUser)&&"".equals(replyUserId)){
                                            continue;
                                        }
                                        String replyUserPhoto = replyEntities.get(i).select(".userPic").select("img").get(0).attr("src");
                                        //System.out.println("回复者头像："+replyUserPhoto);
                                        String replyContent2 = replyInfo.select(".msgCnt").select(".content").get(0).html();
                                        //System.out.println("回复内容："+replyContent2);
                                        String replyTimeString=replyInfo.select(".pubInfo").select(".cNote").select("a").attr("rel");
                                        Date replyDate = new Date(Long.parseLong(replyTimeString+"000"));
                                        SimpleDateFormat localSimpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
                                        String replyTime =localSimpleDateFormat.format(replyDate);
                                        //System.out.println("回复时间："+replyTime);
                                        //String replyUrl = url + "#" + replyEntities.get(i).id();
                                        // System.out.println("回复url："+replyUrl);
                                        JSONArray reply_img_urls=new JSONArray();
                                        Elements replyImgElements=replyInfo.select(".msgCnt").select(".content").get(0).select("img");
                                        for(int imgIndex=0;imgIndex<replyImgElements.size();imgIndex++){
                                            org.jsoup.nodes.Element imgElement=replyImgElements.get(imgIndex);
                                            String img_url=imgElement.absUrl("crs");
                                            reply_img_urls.add(img_url);

                                        }
                                        reply.put("publish_user", (replyUser));
                                        reply.put("publish_user_id", replyUserId);
                                        reply.put("publish_user_photo", replyUserPhoto);
                                        reply.put("content", FileUtils.filterImg(replyContent2));
                                        reply.put("img_urls",reply_img_urls);
                                        reply.put("publish_time", replyTime);
                                        //reply.put("url", replyUrl);
                                        reply.put("parent_id", parent_id);
                                        reply.put("ancestor_id", parent_id);
                                        arrayPage.add(reply);

                                    } catch (Exception e) {

                                        continue;// 因为禁言无法获取的情况
                                    }
                                }
                                flag = false;
                            } else if (String.valueOf(pageContent.getStatusCode()).startsWith("3")) {

                            } else if (String.valueOf(pageContent.getStatusCode()).startsWith("4")) {

                            } else if (String.valueOf(pageContent.getStatusCode()).startsWith("5")) {

                            } else {

                            }

                        } catch (Exception e1) {
                            //System.out.println("异常信息：" + e1.getMessage());

                            logger.error(e1.getMessage());
                        }finally{
                            if(flag==false){
                                array.addAll(arrayPage);
                            }
                            nowTime = System.currentTimeMillis();
                        }
                    }

                } catch (Exception e) {
                    logger.error(e.getMessage(),e);
                    continue;
                }
            }
            return array;
        }

        public  Map<String, Object> getUrlParam(String param) {
            Map<String, Object> map = new HashMap<String, Object>(0);
            if (StringUtils.isBlank(param)) {
                return map;
            }
            String[] params = param.split("&");
            for (int i = 0; i < params.length; i++) {
                if(i==0){
                    String[] p = params[i].split("=");
                    String[] p2=p[0].split("[?]");
                    if (p.length == 2) {
                        map.put(p2[1], p[1]);
                    }
                }else{
                    String[] p = params[i].split("=");
                    if (p.length == 2) {
                        map.put(p[0], p[1]);
                    }
                }
            }
            return map;
        }

        public ReplyCrawler(TencentWeiBoDownloader downLoader, ArrayList<Integer> pages, String url, String parent_id,List<String> pageUrlList,String speeches_id) {
            this.downLoader = downLoader;
            this.pages = pages;
            this.url = url;
            this.parent_id = parent_id;
            this.pageUrlList=pageUrlList;
            this.speeches_id=speeches_id;
        }
    }


}
